#!/usr/bin/env python3
"""Bypass Url Parser, made with love by @TheLaluka
A tool that tests MANY url bypasses to reach a 40X protected page.

Usage:
    bypass-url-parser (-u <URL> | -R <file>) [-m <mode>] [-o <outdir>] [-S <level>] [ (-H <header>)...] [-r <num>]
                      [-s <ip>] [--spoofip-replace] [-p <port>] [--spoofport-replace] [-t <threads>] [-T <timeout>]
                      [--request-tls] [--jsonl] [--dump-payloads] [-x <proxy_url>] [-v | -d | -dd]

Program options:
    -u, --url <URL>           URL (path is optional) to run bypasses against
    -R, --request <file>      Load HTTP raw request from a file
    -H, --header <header>     Header(s) to use, format: "Cookie: can_i_haz=fire"
    -m, --mode <mode>         Bypass modes. See 'Bypasser.BYPASS_MODES' in code [Default: all]
    -o, --outdir <outdir>     Output directory for results
    -x, --proxy <proxy_url>   Set a proxy in the format http://proxy_ip:port.
    -S, --save-level <level>  Save results level. From 0 (DISABLE) to 3 (FULL) [Default: 2]
    -s, --spoofip <ip>        IP(s) to inject in ip-specific headers
    -p, --spoofport <port>    Port(s) to inject in port-specific headers
    -r, --retry <num>         Retry attempts of failed requests. Set 0 to disable all retry tentatives [Default: 1]
    -t, --threads <threads>   Scan with N parallel threads [Default: 1]
    -T, --timeout <timeout>   Request times out after N seconds [Default: 5]

General options:
    -h, --help                Show help, you are here :)
    -v, --verbose             Verbose output
    -d, --debug               Show more details like curl commands generated by this tool
    -dd, --debug              Print Debug level 2 (with all classes debug_class output)
    -V, --version             Show version info

Misc options:
    --spoofip-replace         Disable list of default internal IPs in 'http_headers_ip' bypass mode
    --spoofport-replace       Disable list of default internal ports in 'http_headers_port' bypass mode
    --request-tls             Force usage of TLS/HTTPS for the request load with the '-R, --request' option
    --dump-payloads           Print all payloads (curls) generated by this tool.
    --jsonl                   Print results in JSON lines format (pipe command output)

Examples:
    bypass-url-parser -u "http://127.0.0.1/juicy_403_endpoint/" -s 8.8.8.8 -d
    bypass-url-parser -u /path/urls -t 30 -T 5 -H "Cookie: me_iz=admin" -H "User-agent: test"
    bypass-url-parser -R /path/request_file --request-tls -m "mid_paths, end_paths"
"""

from __future__ import annotations

import concurrent.futures
import hashlib
import json
import locale
import logging
import os
import platform
import re
import socket
import subprocess
import sys
import tempfile
from collections import defaultdict
from enum import IntEnum
from pathlib import Path
from queue import Queue
from shlex import join as shlex_join
from shutil import which
from typing import Any, overload
from urllib.parse import ParseResult, urlparse

import coloredlogs
from docopt import docopt

try:
    from bypass_url_parser._version import __version__
except ImportError:
    __version__ = "dev"

logger = logging.getLogger("bup")


class Bypasser:
    class SaveLevel(IntEnum):
        NONE = 0  # Disable output saving
        MINIMAL = 1  # Only save the program log file which contains the results
        PERTINENT = 2  # Save the program log file and pertinent (results) curl responses in separate html files.
        FULL = 3  # Save the program log file and all curl responses in separate html files.

    # Default class values
    REGEX_URL = re.compile(r"^https?://[^/]+", re.IGNORECASE)
    REGEX_PROXY_URL = re.compile(r"^https?://.*:\d{2,5}$", re.IGNORECASE)
    REGEX_REQ_METHOD = re.compile(r"^(\w+)\s.*\sHTTP/\d\.?\d?$", re.IGNORECASE | re.MULTILINE)
    REGEX_REQ_TARGET = re.compile(r"^\w+\s(.*)\sHTTP/\d\.?\d?$", re.IGNORECASE | re.MULTILINE)
    REGEX_REQ_HTTP_VERSION = re.compile(r"^\w+\s.*\sHTTP/(\d|\d\.\d)$", re.IGNORECASE | re.MULTILINE)
    REGEX_REQ_HOST = re.compile(r"^Host:\s(.*)$", re.IGNORECASE | re.MULTILINE)
    REGEX_REQ_CONTENT_TYPE = re.compile(r"^Content-Type:\s(\w+/\w+);?.*$", re.IGNORECASE | re.MULTILINE)
    BYPASS_MODES = {"all", "mid_paths", "end_paths", "http_host", "http_methods", "http_versions", "case_substitution",
                    "unicode", "char_encode", "http_headers_method", "http_headers_scheme", "http_headers_ip",
                    "http_headers_port", "http_headers_url", "user_agent", "misc"}  # Not yet all implemented
    DEFAULT_BINARY_NAME = which("curl")
    DEFAULT_BYPASS_MODE = "all"
    DEFAULT_FILE_ENCODING = "UTF-8"
    DEFAULT_HTTP_VERSION = "0"  # Disabled by default. Lets curl to manage its own version of HTTP by default
    DEFAULT_LOG_FILENAME = "triaged-bypass.log"
    DEFAULT_JSON_FILENAME = "triaged-bypass.json"
    DEFAULT_OUTPUT_DIR = f"{tempfile.TemporaryDirectory().name}-bypass-url-parser"
    DEFAULT_REQUEST_TLS = False  # Use HTTP protocol by default for requests load with the --request option
    DEFAULT_REQUEST_METHOD = "GET"
    DEFAULT_RETRY_NUMBER = 1
    DEFAULT_SAVE_LEVEL = SaveLevel.PERTINENT
    DEFAULT_SPOOF_IP_REPLACE = False
    DEFAULT_SPOOF_PORT_REPLACE = False
    DEFAULT_TIMEOUT = 5
    DEFAULT_THREAD_NUMBER = 10
    DEFAULT_USER_AGENT = \
        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.41 Safari/537.36"

    def __init__(self, config_dict=None, encoding=None, verbose=False, debug=False, debug_class=False, ext_logger=None):
        if not config_dict:
            config_dict = {}

        # Init verbose and/or debug level
        if config_dict and "--verbose" in config_dict.keys() and "--debug" in config_dict.keys():
            self.verbose = config_dict.get("--verbose", False)
            self.debug = False
            self.debug_class = False
            self._init_debug_level(config_dict.get("--debug"))
        else:
            self.verbose = True if verbose or debug or debug_class else False
            self.debug = True if debug or debug_class else False
            self.debug_class = debug_class

        # Get a logger
        if ext_logger:
            self.logger = ext_logger
        else:
            self.logger = Tools.get_new_logger(self.use_classname(), with_colors=True, debug_level=self.debug)

        if self.debug_class:
            self.logger.debug(
                f"Debug level: verbose={self.verbose}, debug={self.debug}, debug_class={self.debug_class}")

        # Init object vars
        self.base_curl = []
        self.user_agent_suffix = ""
        self.curl_items = set()
        self.curl_ips = []
        self.bypass_results = defaultdict(defaultdict)
        self.to_retry_items = set()
        self.clean_output = ""
        self.pbar_queue = Queue(maxsize=1)
        self.url_resolved_ip = ""
        self.encoding = encoding if encoding else Bypasser.DEFAULT_FILE_ENCODING

        # Import HTTP headers payloads
        self.header_http_methods = Tools.load_file_into_memory_list(
            "payloads/header_http_methods.lst", enc_format=self.encoding, clean_filename=False, external_file=False,
            return_str=False, ext_logger=self.logger, debug=self.debug_class)
        self.header_ip_hosts = Tools.load_file_into_memory_list(
            "payloads/header_ip_hosts.lst", enc_format=self.encoding, clean_filename=False, external_file=False,
            return_str=False, ext_logger=self.logger, debug=self.debug_class)
        self.header_ports = Tools.load_file_into_memory_list(
            "payloads/header_ports.lst", enc_format=self.encoding, clean_filename=False, external_file=False,
            return_str=False, ext_logger=self.logger, debug=self.debug_class)
        self.header_proto_schemes = Tools.load_file_into_memory_list(
            "payloads/header_proto_schemes.lst", enc_format=self.encoding, clean_filename=False, external_file=False,
            return_str=False, ext_logger=self.logger, debug=self.debug_class)
        self.header_urls = Tools.load_file_into_memory_list(
            "payloads/header_urls.lst", enc_format=self.encoding, clean_filename=False, external_file=False,
            return_str=False, ext_logger=self.logger, debug=self.debug_class)

        # Import internal bypass payloads
        self.internal_endpaths = Tools.load_file_into_memory_list(
            "payloads/internal_endpaths.lst", enc_format=self.encoding, clean_filename=False, external_file=False,
            return_str=False, ext_logger=self.logger, debug=self.debug_class)
        self.internal_http_methods = Tools.load_file_into_memory_list(
            "payloads/internal_http_methods.lst", enc_format=self.encoding, clean_filename=False, external_file=False,
            return_str=False, ext_logger=self.logger, debug=self.debug_class)
        self.internal_ip_hosts = Tools.load_file_into_memory_list(
            "payloads/internal_ip_hosts.lst", enc_format=self.encoding, clean_filename=False, external_file=False,
            return_str=False, ext_logger=self.logger, debug=self.debug_class)
        self.internal_midpaths = Tools.load_file_into_memory_list(
            "payloads/internal_midpaths.lst", enc_format=self.encoding, clean_filename=False, external_file=False,
            return_str=False, ext_logger=self.logger, debug=self.debug_class)
        self.internal_ports = Tools.load_file_into_memory_list(
            "payloads/internal_ports.lst", enc_format=self.encoding, clean_filename=False, external_file=False,
            return_str=False, ext_logger=self.logger, debug=self.debug_class)
        self.internal_proto_schemes = Tools.load_file_into_memory_list(
            "payloads/internal_proto_schemes.lst", enc_format=self.encoding, clean_filename=False, external_file=False,
            return_str=False, ext_logger=self.logger, debug=self.debug_class)
        self.internal_user_agents = Tools.load_file_into_memory_list(
            "payloads/internal_user_agents.lst", enc_format=self.encoding, clean_filename=False, external_file=False,
            return_str=False, ext_logger=self.logger, debug=self.debug_class)

        # Init properties
        self.binary_name = Bypasser.DEFAULT_BINARY_NAME
        self.http_version = Bypasser.DEFAULT_HTTP_VERSION
        self.user_agent = Bypasser.DEFAULT_USER_AGENT
        self.proxy = config_dict.get("--proxy")
        self.current_bypass_modes = config_dict.get("--mode")
        self.headers = config_dict.get("--header", [])
        self.output_dir = config_dict.get("--outdir")
        self.save_level = config_dict.get("--save-level")
        self.spoof_ip_replace = config_dict.get("--spoofip-replace")  # If False spoof_ips append to existing list
        self.spoof_ips = config_dict.get("--spoofip")
        self.spoof_port_replace = config_dict.get("--spoofport-replace")  # If False spoof_ports append to existing list
        self.spoof_ports = config_dict.get("--spoofport")
        self.threads = config_dict.get("--threads")
        self.timeout = config_dict.get("--timeout")
        self.retry_number = config_dict.get("--retry")
        self.dump_payloads = config_dict.get("--dump-payloads")
        self.jsonl_output = config_dict.get("--jsonl")
        self.request_tls = config_dict.get("--request-tls")
        if config_dict.get("--request"):
            self.request_raw = Tools.load_file_into_memory_list(
                config_dict.get("--request"), enc_format=self.encoding, clean_filename=False, external_file=True,
                return_str=True, ext_logger=self.logger, debug=self.debug_class)
        else:
            self.request_raw = None
            self.urls = config_dict.get("--url")

    # *** Protected methods *** #

    def _build_curl_ips(self, resolved_ip=None):
        """ Build internal IP list from spoof_ips, internal_ip_hosts and the resolved target IP address.
        :param str resolved_ip: Public (or private) IP address related to the url subdomain
        """
        self.curl_ips.clear()
        # Adds user's custom IP addresses (-s, --spoof-ip)
        if self.spoof_ips:
            for spoof_ip in self.spoof_ips:
                if spoof_ip not in self.curl_ips:
                    self.curl_ips.append(spoof_ip)

        # Append mode (by default and in any case if self.spoof_ips is empty)
        if not self.spoof_ip_replace:
            # Internal IP addresses
            for internal_ip in self.internal_ip_hosts:
                if internal_ip not in self.curl_ips:
                    self.curl_ips.append(internal_ip)
            # Public (or private) IP address
            if resolved_ip and resolved_ip not in self.curl_ips:
                self.curl_ips.append(resolved_ip)

    def _init_debug_level(self, level):
        if level:
            self.verbose = True
            if level == 1:
                self.debug = True
                self.debug_class = False
            elif level == 2:
                self.debug = True
                self.debug_class = True
            else:
                error_msg = f"Bad debug number argument value => {level}. Only support 2 level [-d or -dd]"
                raise ValueError(error_msg)
        else:
            self.debug = False
            self.debug_class = False

    def _generate_curls(self, url_obj: ParseResult):
        if self.verbose and not self.dump_payloads:
            self.logger.warning(f"Stage: generate_curls for {url_obj.geturl()} url")

        # Get information from url
        base_url = f"{url_obj.scheme}://{url_obj.netloc}"
        base_path = f"{url_obj.path}{url_obj.query}"
        target_url = url_obj.geturl()
        if not self.dump_payloads:
            self.logger.debug(f"URL {target_url} parsing: base_url={base_url}, base_path={base_path}")

        # Reset curl list
        self.curl_items.clear()

        # Resolves public (or private) IP of target URL
        try:
            self.url_resolved_ip = socket.gethostbyname(str(url_obj.hostname))
        except (OSError, socket.gaierror):
            error_msg = f"Unable to resolve the subdomain '{url_obj.hostname}'. Please check the url or your " \
                        f"host's DNS resolvers"
            self.logger.error(error_msg)
            raise ConnectionError(error_msg)

        # Original request
        cmd = [*self.base_curl, target_url]
        item = CurlItem(url_obj, self.base_curl, cmd, bypass_mode="original_request", target_ip=self.url_resolved_ip,
                        encoding=self.encoding, debug=self.debug, ext_logger=self.logger)
        self.curl_items.add(item)

        # [http_methods] - Custom request methods (-X)
        if any(mode in {"all", "http_methods"} for mode in self.current_bypass_modes):
            for internal_http_method in self.internal_http_methods:
                cmd = [*self.base_curl, "-X", internal_http_method, target_url]
                item = CurlItem(url_obj, self.base_curl, cmd, bypass_mode="http_methods", encoding=self.encoding,
                                target_ip=self.url_resolved_ip, debug=self.debug, ext_logger=self.logger)
                self.curl_items.add(item)

        # [http_versions] - Tests the url with all http versions supported by curl
        if any(mode in {"all", "http_versions"} for mode in self.current_bypass_modes):
            for http_version in CurlItem.CURL_HTTP_VERSIONS:
                cmd = [*self.get_curl_base(forced_http_version=http_version), target_url]
                item = CurlItem(url_obj, self.base_curl, cmd, bypass_mode="http_versions", encoding=self.encoding,
                                target_ip=self.url_resolved_ip, debug=self.debug, ext_logger=self.logger)
                self.curl_items.add(item)

        # [user_agent] - Custom user agents
        if any(mode in {"all", "user_agent"} for mode in self.current_bypass_modes):
            for internal_ua in self.internal_user_agents:
                curl_base = self.get_curl_base(forced_user_agent=internal_ua)
                cmd = [*curl_base, target_url]
                item = CurlItem(url_obj, curl_base, cmd, bypass_mode="user_agent", encoding=self.encoding,
                                target_ip=self.url_resolved_ip, debug=self.debug, ext_logger=self.logger)
                self.curl_items.add(item)

        # [http_headers_method] - Custom methods
        if any(mode in {"all", "http_headers_method"} for mode in self.current_bypass_modes):
            for header_http_method in self.header_http_methods:
                for internal_http_method in self.internal_http_methods:
                    cmd = [*self.base_curl, "-H", f"{header_http_method}: {internal_http_method}", target_url]
                    item = CurlItem(url_obj, self.base_curl, cmd, bypass_mode="http_headers_method", debug=self.debug,
                                    target_ip=self.url_resolved_ip, encoding=self.encoding, ext_logger=self.logger)
                    self.curl_items.add(item)

        # [http_headers_ip] - Custom host injection headers
        if any(mode in {"all", "http_headers_ip"} for mode in self.current_bypass_modes):
            self._build_curl_ips(resolved_ip=self.url_resolved_ip)
            commands = set()
            for header_ip_host in self.header_ip_hosts:
                # Header which takes 1 as value
                if header_ip_host == "X-AppEngine-Trusted-IP-Request":
                    commands.add(tuple([*self.base_curl, "-H", f"{header_ip_host}: 1", target_url]))
                    continue
                # Specific rule for header 'Forwarded: for='
                for ip in self.curl_ips:
                    if header_ip_host == "Forwarded":
                        commands.add(tuple([*self.base_curl, "-H", f"{header_ip_host}: by={ip}", target_url]))
                        commands.add(tuple([*self.base_curl, "-H", f"{header_ip_host}: for={ip}", target_url]))
                        commands.add(tuple([*self.base_curl, "-H", f"{header_ip_host}: host={ip}", target_url]))
                    else:
                        commands.add(tuple([*self.base_curl, "-H", f"{header_ip_host}: {ip}", target_url]))
            # Add items
            for command in commands:
                item = CurlItem(url_obj, self.base_curl, [*command], bypass_mode="http_headers_ip", debug=self.debug,
                                target_ip=self.url_resolved_ip, encoding=self.encoding, ext_logger=self.logger)
                self.curl_items.add(item)

        # [http_headers_scheme] - Custom scheme rewrite with X-Forwarded-Scheme
        if any(mode in {"all", "http_headers_scheme"} for mode in self.current_bypass_modes):
            commands = set()
            for header_proto_scheme in self.header_proto_schemes:
                # Adding non-standard headers that take 'on' value (Ex: Microsoft)
                if header_proto_scheme in {"Front-End-Https", "X-Forwarded-HTTPS", "X-Forwarded-SSL"}:
                    commands.add(tuple([*self.base_curl, "-H", f"{header_proto_scheme}: on", target_url]))
                    continue
                for internal_proto_scheme in self.internal_proto_schemes:
                    if header_proto_scheme == "Forwarded":
                        # Specific rule for header 'Forwarded: proto='
                        commands.add(
                            tuple([*self.base_curl, "-H", f"Forwarded: proto={internal_proto_scheme}", target_url]))
                    else:
                        # Standard headers ending with "-Proto" or "-Scheme"
                        commands.add(tuple(
                            [*self.base_curl, "-H", f"{header_proto_scheme}: {internal_proto_scheme}", target_url]))
            # Add items
            for command in commands:
                item = CurlItem(url_obj, self.base_curl, [*command], bypass_mode="http_headers_scheme",
                                target_ip=self.url_resolved_ip, encoding=self.encoding, debug=self.debug,
                                ext_logger=self.logger)
                self.curl_items.add(item)

        # [http_headers_port] - Custom port rewrite
        if any(mode in {"all", "http_headers_port"} for mode in self.current_bypass_modes):
            commands = set()
            for header_port in self.header_ports:
                if self.spoof_ports:
                    # Custom port(s)
                    for spoof_port in self.spoof_ports:
                        commands.add(tuple([*self.base_curl, "-H", f"{header_port}: {spoof_port}", target_url]))
                if not self.spoof_port_replace:  # False in any case if self.spoof_ports is empty
                    # Internal ports
                    for internal_port in self.internal_ports:
                        commands.add(tuple([*self.base_curl, "-H", f"{header_port}: {internal_port}", target_url]))
                # Add items
                for command in commands:
                    item = CurlItem(url_obj, self.base_curl, [*command], bypass_mode="http_headers_port",
                                    target_ip=self.url_resolved_ip, encoding=self.encoding, debug=self.debug,
                                    ext_logger=self.logger)
                    self.curl_items.add(item)

        # [http_headers_url] - Custom urls injection headers
        if any(mode in {"all", "http_headers_url"} for mode in self.current_bypass_modes):
            commands = set()
            current_path = Path(base_path)
            for header_url in self.header_urls:
                # First variant: Targets the base_url and moves original path to the headers
                commands.add(tuple([*self.base_curl, "-H", f"{header_url}: {base_path}", f"{base_url}/"]))
                # Second variant: Targets the base_url and moves target_url to the headers (only for some headers)
                if any(part in header_url.lower() for part in {"url", "request", "file"}):
                    commands.add(tuple([*self.base_curl, "-H", f"{header_url}: {target_url}", f"{base_url}/"]))
                # Third variant: Keeps the original target_url and go up the parent paths in headers
                for parent in current_path.parents:
                    parent_path = str(parent).replace('\\', '/')
                    commands.add(tuple([*self.base_curl, "-H", f"{header_url}: {parent_path}", target_url]))
                    # Fourth variant: Same as third but with complete url (only for some headers)
                    if any(part in header_url.lower() for part in {"url", "refer"}):
                        commands.add(
                            tuple([*self.base_curl, "-H", f"{header_url}: {base_url}{parent_path}", target_url]))
            # Add items
            for command in commands:
                item = CurlItem(url_obj, self.base_curl, [*command], bypass_mode="http_headers_url", debug=self.debug,
                                target_ip=self.url_resolved_ip, encoding=self.encoding, ext_logger=self.logger)
                self.curl_items.add(item)

        # [mid_paths] - Custom paths with extra-mid-slash
        if any(mode in {"all", "mid_paths"} for mode in self.current_bypass_modes):
            commands = set()
            for idx_slash in range(base_path.count("/")):
                for internal_midpath in self.internal_midpaths:
                    path_post = Tools.replacenth(base_path, "/", f"/{internal_midpath}", idx_slash)
                    commands.add(tuple([*self.base_curl, f"{base_url}{path_post}"]))  # First variant
                    commands.add(tuple([*self.base_curl, f"{base_url}/{path_post}"]))  # Second variant
                    if idx_slash <= 1:
                        continue
                    path_pre = Tools.replacenth(base_path, "/", f"{internal_midpath}/", idx_slash)
                    commands.add(tuple([*self.base_curl, f"{base_url}{path_pre}"]))  # Fist variant
                    commands.add(tuple([*self.base_curl, f"{base_url}/{path_pre}"]))  # Second variant
            # Add items
            for command in commands:
                item = CurlItem(url_obj, self.base_curl, [*command], bypass_mode="mid_paths", debug=self.debug,
                                target_ip=self.url_resolved_ip, encoding=self.encoding, ext_logger=self.logger)
                self.curl_items.add(item)

        # [end_paths] - Add suffix
        if any(mode in {"all", "end_paths"} for mode in self.current_bypass_modes):
            commands: set[tuple[str, ...]] = set()
            separator = "" if (base_path == "/" or base_path.endswith("/")) else "/"
            for internal_endpath in self.internal_endpaths:
                # First variant - 'url/suffix'
                commands.add(tuple([*self.base_curl, f"{url_obj.geturl()}{separator}{internal_endpath}"]))
                # Second variant - 'url/suffix/'
                commands.add(tuple([*self.base_curl, f"{url_obj.geturl()}{separator}{internal_endpath}/"]))
                # Only if base_path otherwise the subdomain will be modified and for any non ^[a-zA-Z] endpath
                if base_path != "/":
                    if not re.search(r"^[a-zA-Z]$", internal_endpath[0]):
                        # Third variant - Add 'suffix'
                        commands.add(tuple([*self.base_curl, f"{url_obj.geturl()}{internal_endpath}"]))
                        # Fourth variant variant - Add 'suffix/'
                        commands.add(tuple([*self.base_curl, f"{url_obj.geturl()}{internal_endpath}/"]))
                # Add items
                for command in commands:
                    item = CurlItem(url_obj, self.base_curl, [*command], bypass_mode="end_paths", debug=self.debug,
                                    target_ip=self.url_resolved_ip, encoding=self.encoding, ext_logger=self.logger)
                    self.curl_items.add(item)

        # Char substitution (character-by-character) bypasses
        abc_indexes = [span.start() for span in re.finditer(r"[a-zA-Z]", base_path)]
        for abc_index in abc_indexes:
            # [case_substitution] - Case-Inversion
            if any(mode in {"all", "case_substitution"} for mode in self.current_bypass_modes):
                char_case = base_path[abc_index]
                char_case = char_case.upper() if char_case.islower() else char_case.lower()
                cmd = [*self.base_curl, f"{base_url}{base_path[:abc_index]}{char_case}{base_path[abc_index + 1:]}"]
                item = CurlItem(url_obj, self.base_curl, cmd, bypass_mode="case_substitution", encoding=self.encoding,
                                target_ip=self.url_resolved_ip, debug=self.debug, ext_logger=self.logger)
                self.curl_items.add(item)

            # [char_encode] - Url-Encoding
            if any(mode in {"all", "char_encode"} for mode in self.current_bypass_modes):
                char_urlencoded = format(ord(base_path[abc_index]), "02x")
                single_encoded_path = f"{base_url}{base_path[:abc_index]}%{char_urlencoded}{base_path[abc_index + 1:]}"
                cmd = [*self.base_curl, single_encoded_path]
                item = CurlItem(url_obj, self.base_curl, cmd, bypass_mode="char_encode", encoding=self.encoding,
                                target_ip=self.url_resolved_ip, debug=self.debug, ext_logger=self.logger)
                self.curl_items.add(item)

                # New [char_encode] - Double URL-Encoding by encoding once more
                double_encoded_path = single_encoded_path.replace(f"%{char_urlencoded}", f"%25{char_urlencoded}")
                cmd = [*self.base_curl, double_encoded_path]
                item = CurlItem(url_obj, self.base_curl, cmd, bypass_mode="char_encode_double", encoding=self.encoding,
                                target_ip=self.url_resolved_ip, debug=self.debug, ext_logger=self.logger)
                self.curl_items.add(item)

                # New [char_encode] - Triple URL-Encoding by encoding once more
                triple_encoded_path = single_encoded_path.replace(f"%{char_urlencoded}", f"%2525{char_urlencoded}")
                cmd = [*self.base_curl, triple_encoded_path]
                item = CurlItem(url_obj, self.base_curl, cmd, bypass_mode="char_encode_triple", encoding=self.encoding,
                                target_ip=self.url_resolved_ip, debug=self.debug, ext_logger=self.logger)
                self.curl_items.add(item)

        # Verbose/debug print
        if self.verbose and not self.dump_payloads:
            self.logger.info(f"Payloads to test: {len(self.curl_items)}")

        # IDEA Generate moooooore with cross products?
        # Not doing for now, so many curls already... :)
        return

    def _progress_bar_callback(self, *args: Any):
        self.iteration = self.pbar_queue.get(timeout=10)  # out =>
        # Log every 50 completed requests
        if self.iteration % 50 == 0:
            self.logger.info(f"Doing: {self.iteration} / {self.total}")
        self.pbar_queue.put(self.iteration + 1)  # <= in

    def _run_curls(self, items):
        """Call multithread curl commands.

        :param set[CurlItem] items: List of item objects
        """
        # Reset progress bar
        self.total = len(items)
        if self.pbar_queue.full():
            self.pbar_queue.get_nowait()
        self.pbar_queue.put(1)

        if self.verbose:
            self.logger.warning(f"Stage: run_curls ({self.threads} threads, timeout {self.timeout}s)")
        with concurrent.futures.ThreadPoolExecutor(max_workers=self.threads) as executor:
            for item in items:
                future = executor.submit(self._run_curl, item)
                future.add_done_callback(self._progress_bar_callback)
            executor.shutdown(wait=True)
        return

    def _run_curl(self, item):
        """Exec curl command.

        :param CurlItem item: Item object
        """
        if self.debug:
            self.logger.info(f"Current: {shlex_join(item.request_curl_cmd)}")
        try:
            process = subprocess.Popen(item.request_curl_cmd, text=True, shell=False, stderr=subprocess.STDOUT,
                                       stdout=subprocess.PIPE, encoding=self.encoding)

            # Get command results
            result = process.communicate(timeout=self.timeout)[0]
            # Successful execution, parse result
            if process.returncode == 0:
                if result:
                    # Apply xxx2unix. Mandatory under Windows/macOS to save curl responses headers in html file
                    # Notes: subprocess.Popen with text=True => universal_newlines=True so maybe useless
                    result = result.replace(os.linesep, "\n")

                    # Store command result in CurlItem object
                    if self.proxy and "HTTP/1.0 200 Connection established" in result:
                        # Delete the additional response proxy header 'HTTP/1.0 200 Connection established'
                        item.response_raw_output = result.split("\n", 2)[2]
                    else:
                        item.response_raw_output = result

                    # Remove from retry list if present
                    if item in self.to_retry_items:
                        self.to_retry_items.remove(item)
                else:
                    error_msg = f'Command "{shlex_join(item.request_curl_cmd)}" succeeds but returns no result'
                    self.logger.error(error_msg)
                    raise ValueError(error_msg)
            # Raise a detailed CalledProcessError when the return code != 0
            else:
                raise subprocess.CalledProcessError(process.returncode, item.request_curl_cmd, output=result)

        except subprocess.CalledProcessError as e:
            if self.debug:
                self.logger.warning(f'Command "{shlex_join(e.cmd)}" returned on-zero error code {e.returncode}: '
                                    f'{e.output}')
            # curl: (92) HTTP/2 stream 0 was not closed cleanly: PROTOCOL_ERROR (err 1)
            if e.returncode == 92:
                # With recent curl versions, can occur with HTTP/2 and the CONNECT method
                if self.debug:
                    self.logger.warning("Curl HTTP/2 with HTTP/1.1 upgrade failed. Force HTTP/1.1 for this "
                                        "request and add to retry list")
                # Force or add HTTP version 1.1 in item curl command
                item.force_http_version("1.1")

                # Add modified item in retry list
                self.to_retry_items.add(item)
            else:
                self.to_retry_items.add(item)

        except subprocess.TimeoutExpired as e:
            if self.debug:
                self.logger.warning(f'Command "{shlex_join(e.cmd)}" timed out: {e.output}')
            self.to_retry_items.add(item)

    def _save_results(self, url_obj):
        if self.save_level != self.SaveLevel.NONE:
            results_path = Tools.get_path_with_url(self.output_dir, url_obj) if len(self.urls) > 1 else self.output_dir
            json_items = {
                "url": url_obj.geturl(),
                "bypass_modes": ', '.join(self.current_bypass_modes),
                "results_path": results_path,
                "results": []
            }
            # Output_directory definition and creation
            outdir = self.output_dir
            try:
                # If multiple URLs, add one subdirectory by url
                if len(self.urls) > 1:
                    # Format: output_dir / https - www.tld.com[-port -] - path - endpoint
                    outdir = Tools.get_path_with_url(self.output_dir, url_obj)
                # Create directory
                if Tools.is_exist_directory(outdir, force_create=True):
                    if self.debug_class:
                        self.logger.debug(f"Output directory '{outdir}{Tools.SEPARATOR}' exists on system")
            except Exception as e:
                error_msg = f"Error while creating output directory '{outdir}{Tools.SEPARATOR}': {e}"
                self.logger.error(error_msg)
                raise ValueError(error_msg)

            # SaveLevel.FULL - Save all items
            if self.save_level >= self.SaveLevel.FULL:
                for item in self.curl_items:
                    # Save only completed requests
                    if item not in self.to_retry_items:
                        if not item.save(outdir, force_output_dir_creation=False):
                            self.logger.warning(f"Error when saving {outdir}{Tools.SEPARATOR}{item.filename} file.")
                        else:
                            # Add curl item json representation
                            json_items["results"].append(json.loads(item.to_json(with_html_filename=True)))
                if self.verbose:
                    self.logger.info(f"All curl responses were saved in the '{outdir}{Tools.SEPARATOR}' directory")

            # SaveLevel.PERTINENT - Save only results items (single and first element of each group)
            elif self.save_level >= self.SaveLevel.PERTINENT:
                if self.bypass_results[url_obj]:
                    for url, item_lst in self.bypass_results[url_obj].items():
                        # Save pertinent curl items as individual HTML file
                        if not item_lst[0].save(outdir, force_output_dir_creation=False):
                            self.logger.warning(
                                f"Error when saving {outdir}{Tools.SEPARATOR}{item_lst[0].filename} file.")
                        else:
                            # Add curl item json representation
                            json_items["results"].append(json.loads(item_lst[0].to_json(with_html_filename=True)))
                    if self.verbose:
                        self.logger.info(f"Only relevant curl responses (results) were saved in the "
                                         f"'{outdir}{Tools.SEPARATOR}' directory")
                else:
                    self.logger.warning("No output to save")

            # Batcat - Starting at SaveLevel.PERTINENT, IOW no html files, no batcat command
            inspect_cmd = ""
            if self.save_level >= self.SaveLevel.PERTINENT and Tools.IS_LINUX:
                # Get first item filename of each group
                filename_lst = []
                for out_key, item_lst in self.bypass_results[url_obj].items():
                    filename = item_lst[0].filename
                    if filename not in filename_lst:
                        filename_lst.append(filename)
                # Build and output batcat command
                if len(self.clean_output) > 1:
                    inspect_cmd = f"echo {outdir}{Tools.SEPARATOR}{{{','.join(filename_lst)}}} | xargs batcat"
                else:
                    inspect_cmd = f"echo {outdir}{Tools.SEPARATOR}{filename_lst} | xargs batcat"

                self.logger.info(f"Also, inspect them manually with batcat:\n{inspect_cmd}")

            # Logfile - Starting at SaveLevel.MINIMAL
            if self.save_level >= self.SaveLevel.MINIMAL:
                log_file = f"{outdir}{Tools.SEPARATOR}{Bypasser.DEFAULT_LOG_FILENAME}"
                with open(log_file, mode="w", encoding=self.encoding) as file:
                    file.write(f"Bypass results for '{url_obj.geturl()}' url:\n")
                    file.write(f"{self.clean_output}\n")
                    if self.save_level >= self.SaveLevel.PERTINENT and inspect_cmd:
                        file.write(f"{inspect_cmd}")
                if self.verbose:
                    self.logger.info(f"Program log file which contains the results saved in {log_file}")

            # Json logfile - Starting at SaveLevel.PERTINENT
            if self.save_level >= self.SaveLevel.PERTINENT:
                json_file = f"{outdir}{Tools.SEPARATOR}{Bypasser.DEFAULT_JSON_FILENAME}"
                with open(json_file, mode="w", encoding=self.encoding) as file:
                    logger.info(f"Save JSON results for '{url_obj.geturl()}' in {json_file}")
                    file.write(json.dumps(json_items, indent=2))
        else:
            if self.debug_class:
                self.logger.debug("No saving any output: SaveLevel.NONE")

    def _output_results(self, url_obj, jsonl_output=False, silent_mode=False) -> defaultdict[list]:
        # Create a new defaultdict(list) of aggregated items and reset output
        grouped_curl_items = defaultdict(list)
        self.clean_output = ""

        # Parse all completed curl items
        for item in self.curl_items:
            # Results aggregating
            if item.response_raw_output:
                key_for_unicity = item.get_formatted_response(with_content_length=False, trunk_redirect_url=True)
                if item not in grouped_curl_items[key_for_unicity]:
                    grouped_curl_items[key_for_unicity].append(item)

        # Output program result
        if grouped_curl_items:
            # Get results and store (for the program log file) program output.
            multiple_urls = True if len(self.urls) > 1 else False
            with_filename = True if self.save_level >= self.SaveLevel.PERTINENT else False
            self.clean_output = \
                Bypasser.get_results_from_grouped_items(
                    url_obj, grouped_curl_items, header_line=True, with_filename=with_filename,
                    jsonl_format=jsonl_output, jsonl_output_dir=self.output_dir, jsonl_multiple_urls=multiple_urls,
                    ext_logger=self.logger, verbose=self.verbose)
            # Print results
            if not silent_mode:
                if self.jsonl_output:
                    # JSON-lines output (stdout)
                    print(self.clean_output, flush=True)
                else:
                    # Normal output in logger (stderr)
                    self.logger.info(f"\n{self.clean_output}")

        return grouped_curl_items

    def _parse_raw_request(self) -> None:
        """ Parse raw request if present to define headers and url.

        Called by get_curl_base and keep the prevalence of existing headers. So it's possible to override a raw request
        header with '-H' argument(s)
        """
        if self.request_raw:
            # Ignore raw request 'Accept-Encoding' header which causes an encoding error in result parsing (ex: gzip)
            forbidden_header = re.compile(r"Accept-Encoding", re.IGNORECASE)
            try:
                # Parse request headers
                for elt in self.request_headers.split("\n")[1:]:
                    key, value = elt.split(":", 1)
                    # To keep prevalence of the -H argument(s), do not delete existing entries
                    if key not in self.headers and not forbidden_header.search(key):
                        self.headers[key] = value.strip()

                # Define HTTP version
                self.http_version = str(Bypasser.REGEX_REQ_HTTP_VERSION.search(self.request_headers).group(1))

                # Define scheme & URL
                request_target = Bypasser.REGEX_REQ_TARGET.search(self.request_headers).group(1)
                base_host = Bypasser.REGEX_REQ_HOST.search(self.request_headers).group(1)
                if self.request_tls:
                    self.urls = f"https://{base_host}{request_target}"
                else:
                    self.urls = f"http://{base_host}{request_target}"

            except AttributeError:
                error_msg = f"Please check that the raw request contains a valid HTTP method and a host " \
                            f"header:\n{self.request_headers}"
                self.logger.error(error_msg)
                raise ValueError(error_msg)

    def _retry_failed_commands(self):
        """Retry/Resend curl commands against failed items."""
        if self.to_retry_items:
            if self.retry_number > 0:
                # Save original threads number and timeout
                original_threads = self.threads
                original_timeout = self.timeout

                retry_count = 0
                while len(self.to_retry_items) > 0:
                    if retry_count >= self.retry_number:
                        break
                    # First retry rounds: threads / 2 and timeout * 2
                    if retry_count < self.retry_number - 1:
                        self.threads = int(round(self.threads / 2))
                        self.timeout = self.timeout * 2
                    # Last round 1 thread / +10s timeout
                    else:
                        self.threads = 1
                        self.timeout = self.timeout + 10
                        # Last round - adjust retry_count for logging
                        if retry_count + 1 == self.retry_number:
                            retry_count = self.retry_number - 1

                    self.logger.info(f"Retry ({retry_count + 1}/{self.retry_number}) the '{len(self.to_retry_items)}' "
                                     f"failed curl commands with {self.threads} threads and {self.timeout}s timeout")

                    retry_count += 1
                    self._run_curls(self.to_retry_items)

                # Restore original threads number and timeout
                self.threads = original_threads
                self.timeout = original_timeout
            # Failed request / retry_number = 0
            else:
                self.logger.warning(f"'{len(self.to_retry_items)}' curl requests failed and were lost for "
                                    f"results. Retry mode is disabled")
        # No failed requests
        else:
            self.logger.debug("Each request has reached its target. No need for retry")

    # *** Public methods *** #

    def dump_bypasser_payloads(self, show_bypass_mode=True, show_full_cmd=False) -> str:
        if show_full_cmd:
            if show_bypass_mode:
                return "\n".join(sorted([f"[{_.bypass_mode}] {str(_.request_curl_cmd)}" for _ in self.curl_items]))
            else:
                return "\n".join(sorted([f"{str(_.request_curl_cmd)}" for _ in self.curl_items]))
        else:
            if show_bypass_mode:
                return "\n".join(sorted([f"[{_.bypass_mode}] {str(_.request_curl_payload)}" for _ in self.curl_items]))
            else:
                return "\n".join(sorted([f"{str(_.request_curl_payload)}" for _ in self.curl_items]))

    def get_curl_base(self, forced_user_agent=None, forced_http_version=None, debug=False) -> list[str]:
        """Build curl base command.

        User-agent prevalence rule : force_user_agent > -H "User-agent: xxx" > -R req_file UA > DEFAULT_USER_AGENT

        Note: place all based elements before '--path-as-is' which is key to split the base command from payload(s)

        :param str forced_user_agent: Forced user-agent
        :param str forced_http_version: Forced HTTP version
        :param bool debug: If True, log the obtained base command at the end of this method
        """
        # Optional raw request parsing
        self._parse_raw_request()

        # Build base curl command
        base_curl = [self.binary_name, "-sS", "-kgi"]

        # Request method
        if self.request_method != "GET":
            base_curl.extend(["-X", self.request_method])

        # HTTP version
        if self.http_version != "0" and not forced_http_version:
            base_curl.append(f"--http{self.http_version}")

        # HTTP proxy
        if self.proxy:
            base_curl.extend(["-x", self.proxy])

        # Custom headers
        for key, value in self.headers.items():
            if key.lower() == "user-agent":
                self.user_agent = value
            else:
                base_curl.extend(["-H", f"{key}: {value}"])

        # Optional raw request data part
        if self.request_raw and self.request_datas:
            base_curl.extend(["-d", f'{self.request_datas}'])

        # User-Agent / forced_user_agent & Pivot option
        if not forced_user_agent:
            # User-agent is a base option
            base_curl.extend(["-H", f"User-Agent: {self.user_agent}{self.user_agent_suffix}"])
            base_curl.append("--path-as-is")
        else:
            base_curl.append("--path-as-is")
            # User-agent is a payload
            base_curl.extend(["-H", f"User-Agent: {forced_user_agent}{self.user_agent_suffix}"])

        # Forced HTTP version
        if forced_http_version:
            if forced_http_version in CurlItem.CURL_HTTP_VERSIONS:
                base_curl.append(f"--http{forced_http_version}")
            else:
                error_msg = f"Unknown HTTP version {forced_http_version} Must be in '{CurlItem.CURL_HTTP_VERSIONS}'"
                self.logger.error(error_msg)
                raise ValueError(error_msg)

        if debug:
            self.logger.debug(f"Base curl command: {' '.join(shlex_join(base_curl))}")

        return base_curl

    def run(self, urls=None, raw_request=None, raw_request_tls=None, silent_mode=False) \
            -> defaultdict[ParseResult, defaultdict]:
        """ Executes and processes curl requests.

        :param any urls: Target base URL(s). Could be str / str list separated by comma / list / filename
        :param str raw_request: Use a complete raw request as target instead of simple url
        :param bool raw_request_tls: Force usage of TLS/HTTPS for the request load with raw_request parameter
        :param bool silent_mode: Disable result printing on STDOUT
        :return: All results produced by this method. Useful in library mode
        """
        # Target URLs or request can be defined in object initialization or here
        if raw_request_tls:
            self.request_tls = raw_request_tls
        if urls and raw_request:
            error_msg = "Please make a choice between url(s) and raw request"
            self.logger.error(error_msg)
            raise ValueError(error_msg)
        elif urls and not raw_request:
            self.urls = urls
        elif not urls and raw_request:
            self.request_raw = raw_request

        # Rebuild curl base command
        self.base_curl = self.get_curl_base(debug=self.debug_class)

        if not self.urls:
            error_msg = "Can't find any valid target url"
            self.logger.error(error_msg)
            raise ValueError(error_msg)

        if self.debug_class:
            self.logger.debug(f"Bypasser configuration:\n{self}")

        # Prepare and run curl commands
        self.bypass_results.clear()
        for url_obj in self.urls:
            # Reset previous results
            self.curl_items.clear()
            self.to_retry_items.clear()

            # Generate curl items and command
            try:
                self._generate_curls(url_obj)
            except ConnectionError:
                self.logger.info(f"URL '{url_obj.geturl()}' was ignored")
                continue

            # Just print payloads if self.dump_payloads
            if self.dump_payloads:
                if self.verbose:
                    print(f"\n{self.use_classname()} has generated {len(self.curl_items)} payloads "
                          f"for '{url_obj.geturl()}' url:")
                sys.stdout.buffer.write(
                    self.dump_bypasser_payloads(show_bypass_mode=True, show_full_cmd=self.debug).encode(self.encoding))
                continue

            # Send curl commands
            if not self.verbose and not self.debug and not self.debug_class:
                self.logger.warning(f"Trying to bypass '{url_obj.geturl()}' url ({len(self.curl_items)} payloads)...")
            self._run_curls(self.curl_items)

            # Retry failed curl requests
            self._retry_failed_commands()

            # Show results
            if self.jsonl_output and self.save_level >= self.SaveLevel.MINIMAL:
                self._output_results(url_obj, jsonl_output=self.jsonl_output, silent_mode=silent_mode)
                # Keep original output for triaged-bypass.log file
                current_result = self._output_results(url_obj, jsonl_output=False, silent_mode=True)
            else:
                current_result = self._output_results(url_obj, jsonl_output=self.jsonl_output, silent_mode=silent_mode)

            if url_obj not in self.bypass_results.keys():
                self.bypass_results[url_obj] = current_result

            # Save results (Curl request/responses and program logfile)
            self._save_results(url_obj)

        # Return global results dict for library mode
        return self.bypass_results

    @staticmethod
    def get_results_from_grouped_items(url_obj: ParseResult, grouped_curl_items: defaultdict[list], header_line=True,
                                       with_filename=True, filter_sc=None, jsonl_format=False, jsonl_output_dir="",
                                       jsonl_multiple_urls=False, ext_logger=None, verbose=False) -> str:
        """Ungroup and return string from aggregated curl items.

        When the Bypasser is used as a library, this method is useful to retrieve the aggregated results in a string,
        as usually returned by the program. See the commented code in the main() function for an example of use.

        :param ParseResult url_obj: The target URL object of the curl command
        :param defaultdict[list] grouped_curl_items: Aggregated curl items. Result of _output_results() method
        :param bool header_line: If True return result headers: '[#####] [bypass_method] [payload] => [status_code]...'
        :param bool with_filename: If True return (item.filename) as end of output line
        :param list filter_sc: List of status codes to filter from results. Ex: [403,405,400]
        :param bool jsonl_format: Format and print results as JSON lines. Defaults to True.
        :param str jsonl_output_dir: Output directory where HTML files are stored (default: "")
        :param bool jsonl_multiple_urls: Adds URL to 'jsonl_output_dir' (when len(self.url) > 1 in Bypasser).
        :param logger ext_logger: Specify your own logger for verbose output (default: None) (Optional)
        :param bool verbose: Show verbose information for this method
        :return: Aggregated results
        """
        clean_output = ""
        if grouped_curl_items:
            filter_status_codes = filter_sc if filter_sc else []
            if ext_logger and verbose and not jsonl_format:
                ext_logger.warning(f"Triaged results & distinct pages for '{url_obj.geturl()}' url:")
            # Build output
            if header_line and not jsonl_format:
                if with_filename:
                    clean_output += f"{CurlItem.get_formatted_item_header()} (filename)\n"
                else:
                    clean_output += f"{CurlItem.get_formatted_item_header()}\n"
            for out_key, item_lst in grouped_curl_items.items():
                item_count = len(item_lst)
                if not jsonl_format:
                    if item_count == 1:
                        if item_lst[0].response_status_code not in filter_status_codes:
                            if with_filename:
                                clean_output += \
                                    f"[SINGLE] {item_lst[0].get_formatted_item()} ({item_lst[0].filename})\n"
                            else:
                                clean_output += f"[SINGLE] {item_lst[0].get_formatted_item()}\n"
                    else:
                        if item_lst[0].response_status_code not in filter_status_codes:
                            if with_filename:
                                clean_output += f"[GROUP ({item_count})] {item_lst[0].get_formatted_item()} " \
                                                f"({item_lst[0].filename})\n"
                            else:
                                clean_output += f"[GROUP ({item_count})] {item_lst[0].get_formatted_item()}\n"
                else:
                    if item_lst[0].response_status_code not in filter_status_codes:
                        if with_filename:
                            json_line = item_lst[0].to_json(
                                result_path=jsonl_output_dir, result_path_with_url=jsonl_multiple_urls,
                                result_count=item_count, with_url=jsonl_format)
                        else:
                            json_line = item_lst[0].to_json(
                                result_path=None, result_count=item_count, with_url=jsonl_format)
                        clean_output = f"{clean_output}{json_line}\n"
        else:
            if ext_logger and verbose:
                ext_logger.warning(f"Failed to find any valid response for '{url_obj.geturl()}' url")

        return clean_output.rstrip("\n")

    # *** Properties ***#

    @property
    def binary_name(self) -> str:
        return self._binary_name

    @binary_name.setter
    def binary_name(self, value):
        if value:
            # Transform relative path to absolute
            binary = os.path.join(os.path.realpath(os.path.dirname(value)), os.path.basename(value))
        else:
            binary = Bypasser.DEFAULT_BINARY_NAME
        # Check presence of curl binary
        if binary is None or not (os.path.isfile(binary) or os.path.islink(binary)):
            error_msg = "Program curl not found, install it and ensure it's within your PATH"
            self.logger.error(error_msg)
            raise FileNotFoundError(error_msg)
        else:
            self._binary_name = binary

    @property
    def current_bypass_modes(self) -> set[str]:
        return self._current_bypass_modes

    @current_bypass_modes.setter
    def current_bypass_modes(self, modes_lst):
        self._current_bypass_modes = set()
        if not modes_lst:
            self._current_bypass_modes.add(Bypasser.DEFAULT_BYPASS_MODE)
        else:
            for mode in Tools.get_list_from_generic_arg(
                    modes_lst, arg_name="bypass_mode", stdin_support=True, comma_string_support=True,
                    enc_format=self.encoding, ext_logger=self.logger, debug=self.debug_class):
                if mode in Bypasser.BYPASS_MODES:
                    self._current_bypass_modes.add(mode)
                else:
                    self.logger.warning(f"Unknown bypass mode {mode} was ignored. Must be in {Bypasser.BYPASS_MODES}")
            if "all" in self._current_bypass_modes:
                self._current_bypass_modes = {"all"}
                if self.debug_class:
                    self.logger.debug("'all' was found in custom bypass mode list. Only this value will be kept")
            # If all BYPASS_MODES was ignored
            if not self._current_bypass_modes:
                error_msg = f"Can't find any valid bypass mode. Need at least one value in {Bypasser.BYPASS_MODES}"
                self.logger.error(error_msg)
                raise ValueError(error_msg)

    @property
    def headers(self) -> dict:
        return self._headers

    @headers.setter
    def headers(self, value):
        try:
            self._headers = {}
            if value:
                for header in Tools.get_list_from_generic_arg(
                        value, arg_name="header", stdin_support=False, comma_string_support=False,
                        enc_format=self.encoding, ext_logger=self.logger, debug=self.debug_class):
                    key, value = header.split(":", 1)
                    self._headers[key] = value.strip()
        except ValueError as e:
            if "not enough values to unpack" in str(e):
                error_msg = "Custom headers parsing error: missing ':' in one header. Please use 'header: val' format"
                self.logger.error(error_msg)
                raise ValueError(error_msg)
        except Exception as e:
            error_msg = f"Custom headers parsing error: {e}"
            self.logger.error(error_msg)
            raise ValueError(error_msg)

    @property
    def output_dir(self) -> str:
        return self._output_dir

    @output_dir.setter
    def output_dir(self, value):
        try:
            self._output_dir = Bypasser.DEFAULT_OUTPUT_DIR
            if value:
                # Transform if needed relative path to absolute
                self._output_dir = os.path.join(os.path.realpath(os.path.dirname(value)), os.path.basename(value))
            if Tools.is_exist_directory(self._output_dir, force_create=False):
                error_msg = f"The output directory already exists: {self._output_dir}"
                self.logger.error(error_msg)
                raise ValueError(error_msg)
        except Exception as e:
            error_msg = f"Custom output_dir parsing error: {e}"
            self.logger.error(error_msg)
            raise ValueError(error_msg)

    @property
    def http_version(self) -> str:
        return self._http_version

    @http_version.setter
    def http_version(self, value) -> None:
        """Set HTTP version used in curl base command.

        Reference: https://everything.curl.dev/http/versions
        CurlItem.CURL_HTTP_VERSIONS: {"0.9", "1.0", "1.1", "2", "2-prior-knowledge"}

        Set to "0" to disable the HTTP version argument in curl and let it handle requests with its default version

        :param str value: HTTP version. Must be "0" or value in CurlItem.CURL_HTTP_VERSIONS
        """
        try:
            self._http_version = Bypasser.DEFAULT_HTTP_VERSION
            if value and str(value) != "0":
                if str(value) in CurlItem.CURL_HTTP_VERSIONS:
                    self._http_version = str(value)
                else:
                    error_msg = f"Unknown HTTP version {str(value)}. Must be in '{CurlItem.CURL_HTTP_VERSIONS}'"
                    self.logger.error(error_msg)
                    raise ValueError(error_msg)
        except Exception as e:
            error_msg = f"Error when setting custom http version: {e}"
            self.logger.error(error_msg)
            raise ValueError(error_msg)

    @property
    def request_content_type(self) -> str | None:
        if self.request_headers:
            content_type = Bypasser.REGEX_REQ_CONTENT_TYPE.search(self.request_headers)
            if content_type:
                return content_type.group(1)
            else:
                return None
        else:
            self.logger.warning("Please define request_raw property before trying to access its headers values.")
            return None

    @property
    def request_datas(self) -> str | None:
        if self.request_raw:
            split_request = self.request_raw.split("\n\n")
            # When data part is present
            if len(split_request) >= 2:
                # Strip useless '\n' for curl command debug
                request_data = split_request[1].replace("\n", "")
                # If JSON, format datas for curl
                if request_data:
                    if re.match(r"application/json", self.request_content_type, re.IGNORECASE):
                        return json.dumps(json.loads(request_data), separators=(',', ':'))
                    else:
                        return request_data
                else:  # Empty data part
                    return None
            else:  # The request doesn't contain a minimum of two '\n'
                return None
        else:
            self.logger.warning("Please define request_raw property before trying to access its datas.")
            return None

    @property
    def request_headers(self) -> str | None:
        if self.request_raw:
            return self.request_raw.split("\n\n")[0]
        else:
            self.logger.warning("Please define request_raw property before trying to access its headers.")
            return None

    @property
    def request_method(self) -> str:
        if self.request_raw:
            method = Bypasser.REGEX_REQ_METHOD.search(self.request_headers)
            if method:
                return method.group(1).upper()
            else:
                error_msg = f"Custom raw request seems not having a method line in its headers:\n{self.request_headers}"
                self.logger.error(error_msg)
                raise ValueError(error_msg)
        else:
            return Bypasser.DEFAULT_REQUEST_METHOD

    @property
    def request_raw(self) -> str:
        return self._request_raw

    @request_raw.setter
    def request_raw(self, value):
        try:
            if value:
                if "\r" in value:
                    self._request_raw = value.replace("\r\n", "\n")  # Windows txt editor
                    self._request_raw = self.request_raw.replace("\r", "\n")  # Mac OS <= 9
                else:
                    self._request_raw = value
            else:
                self._request_raw = None
        except Exception as e:
            error_msg = f"Custom curl raw output parsing error. {e}"
            self.logger.error(error_msg)
            self.logger.error(repr(value))
            raise ValueError(error_msg)

    @property
    def request_tls(self) -> bool:
        return self._request_tls

    @request_tls.setter
    def request_tls(self, value):
        self._request_tls = Bypasser.DEFAULT_REQUEST_TLS
        if value:
            self._request_tls = value

    @property
    def retry_number(self) -> int:
        return self._retry_number

    @retry_number.setter
    def retry_number(self, value):
        try:
            self._retry_number = int(Bypasser.DEFAULT_RETRY_NUMBER)
            if value:
                self._retry_number = int(value)
        except Exception as e:
            error_msg = f"Invalid number of retry value: {e}"
            self.logger.error(error_msg)
            raise ValueError(error_msg)
        if self._retry_number < 0:
            error_msg = f"Retry number must be positive or equal to zero: '{self._retry_number}'"
            self.logger.error(error_msg)
            raise ValueError(error_msg)

    @property
    def save_level(self) -> int:
        return self._save_level

    @save_level.setter
    def save_level(self, value):
        try:
            self._save_level = Bypasser.DEFAULT_SAVE_LEVEL
            if value is not None:
                self._save_level = int(value)
        except Exception as e:
            error_msg = f"Invalid number of save_level value: {e}"
            self.logger.error(error_msg)
            raise ValueError(error_msg)
        if self._save_level not in [item.value for item in self.SaveLevel]:
            error_msg = f"Unknown save_level: '{self._save_level}'. Must be in the range of 0 (DISABLE) to 3 (FULL)"
            self.logger.error(error_msg)
            raise ValueError(error_msg)

    @property
    def spoof_ips(self) -> list[str]:
        return self._spoof_ips

    @spoof_ips.setter
    def spoof_ips(self, value):
        try:
            self._spoof_ips = []
            if value:
                for ip in Tools.get_list_from_generic_arg(
                        value, arg_name="spoofip", stdin_support=True, comma_string_support=True,
                        enc_format=self.encoding, ext_logger=self.logger, debug=self.debug_class):
                    if ip not in self._spoof_ips:
                        self._spoof_ips.append(ip)
            # Cancel the possible replace_mode that could block internal ip list in [http_headers_ip]
            if not self._spoof_ips:
                self.spoof_ip_replace = False

        except Exception as e:
            error_msg = f"Custom spoofip parsing error: {e}"
            self.logger.error(error_msg)
            raise ValueError(error_msg)

    @property
    def spoof_ip_replace(self) -> bool:
        return self._spoof_ip_replace

    @spoof_ip_replace.setter
    def spoof_ip_replace(self, value):
        self._spoof_ip_replace = Bypasser.DEFAULT_SPOOF_IP_REPLACE
        if value:
            self._spoof_ip_replace = value

    @property
    def spoof_ports(self) -> list[int]:
        return self._spoof_ports

    @spoof_ports.setter
    def spoof_ports(self, value):
        try:
            self._spoof_ports = []
            if value:
                for port in Tools.get_list_from_generic_arg(
                        value, arg_name="spoofport", stdin_support=True, comma_string_support=True,
                        enc_format=self.encoding, ext_logger=self.logger, debug=self.debug_class):
                    if int(port) not in self._spoof_ports:
                        self._spoof_ports.append(int(port))
            # Cancel the possible replace_mode that could block internal port list in [http_headers_port]
            if not self._spoof_ports:
                self.spoof_port_replace = False
        except Exception as e:
            error_msg = f"Invalid port number value: {e}"
            self.logger.error(error_msg)
            raise ValueError(error_msg)

    @property
    def spoof_port_replace(self) -> bool:
        return self._spoof_port_replace

    @spoof_port_replace.setter
    def spoof_port_replace(self, value):
        self._spoof_port_replace = Bypasser.DEFAULT_SPOOF_PORT_REPLACE
        if value:
            self._spoof_port_replace = value

    @property
    def threads(self) -> int:
        return self._thread

    @threads.setter
    def threads(self, value):
        try:
            self._thread = int(Bypasser.DEFAULT_THREAD_NUMBER)
            if value:
                self._thread = int(value)
        except Exception as e:
            error_msg = f"Invalid number of threads value: {e}"
            self.logger.error(error_msg)
            raise ValueError(error_msg)
        if self._thread <= 0:
            error_msg = f"Thread number must be positive: '{self._thread}'"
            self.logger.error(error_msg)
            raise ValueError(error_msg)

    @property
    def timeout(self) -> int:
        return self._timeout

    @timeout.setter
    def timeout(self, value):
        try:
            self._timeout = int(Bypasser.DEFAULT_TIMEOUT)
            if value:
                self._timeout = int(value)
        except Exception as e:
            error_msg = f"Invalid timeout value: {e}"
            self.logger.error(error_msg)
            raise ValueError(error_msg)
        if self._timeout <= 0:
            error_msg = f"Timeout value (sec) number must be positive: '{self._timeout}'"
            self.logger.error(error_msg)
            raise ValueError(error_msg)

    @property
    def urls(self) -> list[ParseResult]:
        return self._urls

    @urls.setter
    def urls(self, value):
        try:
            self._urls = []
            if value:
                for url in Tools.get_list_from_generic_arg(
                        value, arg_name="url", stdin_support=True, comma_string_support=True,
                        enc_format=self.encoding, ext_logger=self.logger, debug=self.debug_class):
                    if not Bypasser.REGEX_URL.match(url):
                        error_msg = f"URL {url} was ignored. Must start with http(s):// and contain at least 3 slashes"
                        self.logger.warning(error_msg)
                    else:
                        parsed_url = urlparse(url)
                        if parsed_url not in self._urls:
                            self.urls.append(parsed_url)
        except Exception as e:
            error_msg = f"URL parsing error: {e}"
            self.logger.error(error_msg)
            raise ValueError(error_msg)

    @property
    def proxy(self):
        return self._proxy

    @proxy.setter
    def proxy(self, value):
        try:
            self._proxy = None
            if value:
                if not Bypasser.REGEX_PROXY_URL.match(value):
                    error_msg = f"Proxy URL {value} was ignored. Format: 'http(s)://proxy_ip:port'"
                    self.logger.error(error_msg)
                else:
                    self._proxy = value
        except Exception as e:
            error_msg = f"Custom proxy parsing error: {e}"
            self.logger.error(error_msg)
            raise ValueError(error_msg)

    @property
    def user_agent(self) -> str:
        return self._user_agent

    @user_agent.setter
    def user_agent(self, value):
        try:
            self._user_agent = Bypasser.DEFAULT_USER_AGENT
            if value:
                self._user_agent = value
        except Exception as e:
            error_msg = f"Custom user-agent parsing error: {e}"
            self.logger.error(error_msg)
            raise ValueError(error_msg)

    # *** Class identity, comparison and hashing functions *** #

    @classmethod
    def get_classname(cls):
        """Returns the fully-qualified class name string of this object."""
        return cls.__name__

    def use_classname(self) -> str:
        return self.get_classname()

    def __attrs(self):
        return (self.current_bypass_modes, self.headers, self.spoof_ips, self.threads, self.timeout,
                self.dump_payloads, self.urls, self.user_agent, self.verbose, self.debug, self.debug_class)

    def __eq__(self, other) -> bool:
        """Equality test between objects, returns True if both objects have the same type and same attributes.

        :param Bypasser other: Other object to compare with this
        :return: True is same object, else False
        """
        return other.__class__ == Bypasser and self.__dict__ == other.__dict__

    def __ne__(self, other) -> bool:
        """Define a non-equality test.

        :param Bypasser other: Other object to compare with this
        :return: False is same object, else True
        """
        return not self.__eq__(other)

    def __hash__(self) -> int:
        """Used to get and store a unique object reference in immutable collections.

        :return: Object hash
        """
        return hash(str(self.__dict__))

    def __str__(self) -> str:
        out = ""
        out += f"Url(s): {[url_obj.geturl() for url_obj in self.urls]}\n"
        out += f"Headers(s): {self.headers}\n"
        out += f"Current bypass modes: {self.current_bypass_modes}\n"
        out += f"Threads: {self.threads}\n"
        out += f"Dump Payloads: {self.dump_payloads}\n"
        out += f"Timeout: {self.timeout}\n"
        out += f"Retry number: {self.retry_number}\n"
        out += f"User-agent: {self.user_agent}\n"
        out += f"Proxy: {self.proxy}\n"
        out += f"Spoofip(s): {self.spoof_ips}\n"
        out += f"Spoofip replace: {self.spoof_ip_replace}\n"
        out += f"Spoofport(s): {self.spoof_ports}\n"
        out += f"Spoofport replace: {self.spoof_port_replace}\n"
        out += f"Save level: {self.save_level}\n"
        out += f"Output directory: {self.output_dir}\n"
        out += f"Verbose: {self.verbose}\n"
        out += f"Debug: {self.debug}\n"
        out += f"Debug_class: {self.debug_class}\n"
        out += f"Curl base: {self.base_curl}\n"
        return out


class CurlItem:
    """Utility Class to store request and response elements related to a curl commands

    How to use this class:
      - Init object with at least Url (ParseResult) object, curl base command, final curl cmd and bypass_mode:
        item = CurlItem(url_obj, base_curl_cmd, curl_cmd, "[http_methods]")
      - Send curl command outside this class:
        response = exec_fct(item.self.request_curl_cmd)
      - Pass curl command output to response_raw_output property
        item.response_raw_output = response

    CurlItem exposes the following properties / public variables:
      - request_curl_cmd, request_curl_payload, curl_base_options, target_url, target_ip, bypass_mode
      - response_raw_output, response_headers, response_data, response_content_length, response_content_type,
        response_lines_count, response_redirect_url, response_server_type, response_status_code, response_title
    """
    CURL_HTTP_VERSIONS = {"0.9", "1.0", "1.1", "2", "2-prior-knowledge"}
    DEFAULT_FILE_ENCODING = "UTF-8"
    REGEX_STATUS_CODE = re.compile(r"HTTP.*\s+(\d+)\s+", re.IGNORECASE)
    REGEX_CONTENT_LENGTH = re.compile(r"Content-Length:\s+(\d+)", re.IGNORECASE)
    REGEX_CONTENT_TYPE = re.compile(r"Content-Type:\s+(\w+/\w+)", re.IGNORECASE)
    REGEX_HTTP_VERSION = re.compile(r"(?!--)http[\w.-]*(?<! )", re.IGNORECASE)
    REGEX_REDIRECT_URL = re.compile(r"Location:\s+(.*)", re.IGNORECASE)
    REGEX_SERVER_TYPE = re.compile(r"Server:\s+(.*)", re.IGNORECASE)
    REGEX_TITLE = re.compile(r"<title>(.*?)</title>", re.IGNORECASE)
    REDIRECT_URL_MAX_SIZE = 25

    def __init__(self, target_url: ParseResult, curl_base: list[str], curl_cmd: list[str],
                 bypass_mode: str | None = None, target_ip: str | None = None, encoding=None, debug=False,
                 ext_logger=None):
        """CurlItem object init method

        :param ParseResult target_url: Curl command target url object
        :param list[str] curl_base: Curl base command. Ex: curl -sS -kgi --path-as-is
        :param list[str] curl_cmd: Curl full command. Ex: curl -sS -kgi --path-as-is -H 'xxx' https://target.com/path
        :param str bypass_mode:  Payload description (Optional)
        :param str target_ip: IP address of target url (Optional)
        """
        # Get a logger
        self.debug = debug
        if ext_logger:
            self.logger = ext_logger
        else:
            self.logger = Tools.get_new_logger(self.use_classname(), with_colors=True, debug_level=self.debug)

        # Encoding and request elements #
        self.encoding = encoding if encoding else CurlItem.DEFAULT_FILE_ENCODING
        self.target_url = target_url
        self.target_ip = target_ip
        self.bypass_mode = bypass_mode
        self.curl_base_options = curl_base
        self.request_curl_cmd = curl_cmd

        # Response elements #
        # All the response_xxx attributes are filled automatically when the 'res_raw_output' property is set.
        self.response_raw_output = ""

    # *** Public methods *** #

    def force_http_version(self, target_http_version) -> None:
        """Patch the current item (curl_base_options and request_curl_cmd) to force a specific http version

        :param str target_http_version: Forced http version. Ex: 1.1
        """
        # Security check
        if str(target_http_version) not in CurlItem.CURL_HTTP_VERSIONS:
            error_msg = f"Unknown HTTP version {str(target_http_version)} was ignored. Must be " \
                        f"in '{CurlItem.CURL_HTTP_VERSIONS}'"
            self.logger.error(error_msg)
            raise ValueError(error_msg)
        # Patch HTTP version
        self.curl_base_options = [option for option in self.curl_base_options if not option.startswith("--http")]
        self.curl_base_options.append(f"--http{target_http_version}")

    @staticmethod
    def get_formatted_item_header(separator="=>") -> str:
        """Return formatted curl item as string.

        Ex: [http_methods] [-X 'TRACE'] => [405] [text/html] [524] [12] [70] [405 Not Allowed] []

        :return: Formatted curl item response.
        """
        return f"[#####] [bypass_method] [payload] {separator} [status_code] [content_type] [content_length] " \
               f"[lines_count] [word_counts] [title] [server] [redirect_url]"

    def get_formatted_item(self, separator="=>") -> str:
        """Return formatted curl item as string.

        Ex: [http_methods] [-X 'TRACE'] => [405] [text/html] [524] [12] [70] [405 Not Allowed] [nginx] []

        :return: Formatted curl item response.
        """
        return f"{self.get_formatted_payload()} {separator} {self.get_formatted_response(with_content_length=True)}"

    def get_formatted_payload(self) -> str:
        """Return formatted curl item payload as string.

        :return: Formatted curl item response. Ex: [http_method] [-X 'POST']
        """
        return f"[{self.bypass_mode}] [{self.request_curl_payload}]"

    def get_formatted_response(self, with_content_length=False, trunk_redirect_url=True) -> str:
        """Return formatted curl item response as string

        Format:
         - [Status-code] [Content-type] [Content-length] [Lines-count] [Words-count] [Title] [Server] [Redirect_url]

        Note: The two optional arguments are mainly used when aggregating responses' elements:
         - Removing the content length can be useful for response comparison. Indeed, Lines-count/Words-count
           of response data are usually more reliable
         - Limiting the size of the redirect URL can be useful to prevent it from becoming a discriminant when a
           random token is included in a long redirect URL.

        :param bool with_content_length: If False, remove content-length from response
        :param bool trunk_redirect_url: If True, truncate the size of redirection url
        :return: Formatted curl item response. Ex: [403] [text/html] [548] [13] [68] [403 Forbidden] []
        """
        # Limit the size of the redirect URL (optional)
        redirect_url = self.response_redirect_url
        if redirect_url and trunk_redirect_url:
            redirect_url = self.response_redirect_url[: CurlItem.REDIRECT_URL_MAX_SIZE]

        # Use it for full response output (content_length and full redirect_url)
        if with_content_length:
            return f"[{self.response_status_code}] [{self.response_content_type}] [{self.response_content_length}] " \
                   f"[{self.response_lines_count}] [{self.response_words_count}] [{self.response_title}] " \
                   f"[{self.response_server_type}] [{self.response_redirect_url}]"
        # Use it for response aggregating (no content_length and trunked redirect_url)
        else:
            return f"[{self.response_status_code}] [{self.response_content_type}] [{self.response_lines_count}] " \
                   f"[{self.response_words_count}] [{self.response_title}] [{self.response_server_type}] " \
                   f"[{redirect_url}]"

    def save(self, output_dir, force_output_dir_creation=False) -> bool:
        out_filename = f"{output_dir}{Tools.SEPARATOR}{self.filename}"
        if self.response_raw_output and Tools.is_exist_directory(output_dir, force_create=force_output_dir_creation):
            self.logger.debug(f"Saving html pages and short output in: '{output_dir}{Tools.SEPARATOR}'")
            with open(f"{out_filename}", mode="w", encoding=self.encoding) as file:
                file.write(f"{shlex_join(self.request_curl_cmd)}\n\n{self.response_headers}\n\n{self.response_data}")
            return True
        else:
            return False

    def to_json(self, result_path="", result_path_with_url=False, result_count=0, with_html_filename=False,
                with_url=False) -> str:
        """ Converts a curl item object into a JSON representation.

        :param str result_path: Add 'response_html_path' to json if present
        :param bool result_path_with_url: Adds URL to 'result_path' (when len(self.url) > 1 in Bypasser)
        :param int result_count: If greater than or equal to 1, adds 'result_count' and 'results_tag' to json
        :param bool with_html_filename: Includes 'response_html_filename' to json if True or if result_path is present
        :param bool with_url: Includes 'request_url' to json if True
        :return: JSON representation of the object
        """
        json_repr = {
            "request_curl_cmd": shlex_join(self.request_curl_cmd),
            "request_curl_payload": self.request_curl_payload,
            "response_headers": self.response_headers,
            "response_data": self.response_data,
            "response_status_code": self.response_status_code,
            "response_content_type": self.response_content_type,
            "response_content_length": self.response_content_length,
            "response_lines_count": self.response_lines_count,
            "response_words_count": self.response_words_count,
            "response_title": self.response_title,
            "response_server_type": self.response_server_type,
            "response_redirect_url": self.response_redirect_url
            # "response_html_filename": self.filename
        }
        # Add response HTML path and filename based on options
        if result_path:
            json_repr["response_html_filename"] = self.filename
            json_repr["response_html_path"] = Tools.get_path_with_url(result_path, self.target_url) \
                if result_path_with_url else result_path
        else:
            if with_html_filename:
                json_repr["response_html_filename"] = self.filename

        # Add result count and result tag
        if isinstance(result_count, int) and result_count:
            json_repr["results_count"] = result_count
            json_repr["results_tag"] = "SINGLE" if result_count == 1 else "GROUP"

        # Add request URL if specified
        if with_url:
            json_repr["request_url"] = self.target_url.geturl()

        return json.dumps(json_repr, sort_keys=False)

    # *** Properties *** #

    @property
    def filename(self) -> str:
        return f"bypass-{hashlib.md5(' '.join(map(str, self._curl_cmd)).encode()).hexdigest()}.html"

    @property
    def request_curl_payload(self) -> str:
        return " ".join(self._curl_cmd[self._curl_cmd.index("--path-as-is") + 1:])

    @property
    def request_curl_cmd(self) -> str:
        return self._curl_cmd

    @request_curl_cmd.setter
    def request_curl_cmd(self, value):
        try:
            self._curl_cmd = None
            if value:
                self._curl_cmd = value
        except Exception as e:
            error_msg = f"Custom curl command parsing error. {e}"
            self.logger.error(error_msg)
            raise ValueError(error_msg)

    @property
    def response_data(self) -> str:
        if self.response_raw_output:
            try:
                # Note: Do not use os.linesep here, all OS specific line separators have already been replaced by '\n'.
                return self.response_raw_output.split("\n\n")[1]
            except Exception as e:
                error_msg = f"Unable to return response data, please check the format and redefine " \
                            f"the 'response_raw_output' property {e}"
                self.logger.error(error_msg)
                self.logger.error(repr(self.response_raw_output))
                raise ValueError(error_msg)
        else:
            error_msg = "Please set 'response_raw_output' property before trying to access its data."
            self.logger.error(error_msg)
            raise ValueError(error_msg)

    @property
    def response_headers(self) -> str:
        if self.response_raw_output:
            try:
                # Note: Do not use os.linesep here, all OS specific line separators have already been replaced by '\n'.
                return self.response_raw_output.split("\n\n")[0]
            except Exception as e:
                error_msg = f"Unable to return response headers, please check the format and redefine " \
                            f"the 'response_raw_output' property {e}"
                self.logger.error(error_msg)
                self.logger.error(repr(self.response_raw_output))
                raise ValueError(error_msg)
        else:
            error_msg = "Please set 'response_raw_output' property before trying to access its headers."
            self.logger.error(error_msg)
            raise ValueError(error_msg)

    @property
    def response_status_code(self) -> int:
        if self.response_raw_output:
            try:
                return int(CurlItem.REGEX_STATUS_CODE.search(self.response_headers).group(1))
            except Exception as e:
                error_msg = f"Unable to return response status_code, please check the format and redefine " \
                            f"the 'response_raw_output' property {e}"
                self.logger.error(error_msg)
                self.logger.error(repr(self.response_raw_output))
                raise ValueError(error_msg)
        else:
            error_msg = "Please set 'response_raw_output' property before trying to access its status_code value."
            self.logger.error(error_msg)
            raise ValueError(error_msg)

    @property
    def response_content_length(self) -> int:
        if self.response_raw_output:
            try:
                match_content_length = CurlItem.REGEX_CONTENT_LENGTH.search(self.response_headers)
                return int(match_content_length.group(1).rstrip()) if match_content_length else len(self.response_data)
            except Exception as e:
                error_msg = f"Unable to return response content_length, please check the format and " \
                            f"redefine the 'response_raw_output' property {e}"
                self.logger.error(error_msg)
                self.logger.error(repr(self.response_raw_output))
                raise ValueError(error_msg)
        else:
            error_msg = "Please set 'response_raw_output' property before trying to access its content_length value."
            self.logger.error(error_msg)
            raise ValueError(error_msg)

    @property
    def response_lines_count(self) -> int:
        if self.response_raw_output:
            try:
                return int(self.response_data.count("\n"))
            except Exception as e:
                error_msg = f"Unable to return response lines_count, please check the format and redefine " \
                            f"the 'response_raw_output' property {e}"
                self.logger.error(error_msg)
                self.logger.error(repr(self.response_raw_output))
                raise ValueError(error_msg)
        else:
            error_msg = "Please set 'response_raw_output' property before trying to access its lines_count value."
            self.logger.error(error_msg)
            raise ValueError(error_msg)

    @property
    def response_words_count(self) -> int:
        if self.response_raw_output:
            try:
                return int(self.response_data.count(" "))
            except Exception as e:
                error_msg = f"Unable to return response words_count, please check the format and redefine " \
                            f"the 'response_raw_output' property {e}"
                self.logger.error(error_msg)
                self.logger.error(repr(self.response_raw_output))
                raise ValueError(error_msg)
        else:
            error_msg = "Please set 'response_raw_output' property before trying to access its words_count value."
            self.logger.error(error_msg)
            raise ValueError(error_msg)

    @property
    def response_content_type(self) -> str:
        if self.response_raw_output:
            try:
                match_content_type = CurlItem.REGEX_CONTENT_TYPE.search(self.response_headers)
                return match_content_type.group(1).rstrip() if match_content_type else ""
            except Exception as e:
                error_msg = f"Unable to return response content_type, please check the format and redefine " \
                            f"the 'response_raw_output' property {e}"
                self.logger.error(error_msg)
                self.logger.error(repr(self.response_raw_output))
                raise ValueError(error_msg)
        else:
            error_msg = "Please set 'response_raw_output' property before trying to access its content_type value."
            self.logger.error(error_msg)
            raise ValueError(error_msg)

    @property
    def response_redirect_url(self) -> str:
        if self.response_raw_output:
            try:
                match_redirect_url = CurlItem.REGEX_REDIRECT_URL.search(self.response_headers)
                return match_redirect_url.group(1).rstrip() if match_redirect_url else ""
            except Exception as e:
                error_msg = f"Unable to return response redirect_url, please check the format and redefine " \
                            f"the 'response_raw_output' property {e}"
                self.logger.error(error_msg)
                self.logger.error(repr(self.response_raw_output))
                raise ValueError(error_msg)
        else:
            error_msg = "Please set 'response_raw_output' property before trying to access its redirect_url value."
            self.logger.error(error_msg)
            raise ValueError(error_msg)

    @property
    def response_server_type(self) -> str:
        if self.response_raw_output:
            try:
                match_server_type = CurlItem.REGEX_SERVER_TYPE.search(self.response_headers)
                return match_server_type.group(1).rstrip() if match_server_type else ""
            except Exception as e:
                error_msg = f"Unable to return response server_type, please check the format and redefine " \
                            f"the 'response_raw_output' property {e}"
                self.logger.error(error_msg)
                raise ValueError(error_msg)
        else:
            error_msg = "Please set 'response_raw_output' property before trying to access its server_type value."
            self.logger.error(error_msg)
            raise ValueError(error_msg)

    @property
    def response_title(self) -> str:
        if self.response_raw_output:
            try:
                match_title = CurlItem.REGEX_TITLE.search(self.response_data)
                return match_title.group(1) if match_title else ""
            except Exception as e:
                error_msg = f"Unable to return response title, please check the format and redefine " \
                            f"the 'response_raw_output' property {e}"
                self.logger.error(error_msg)
                raise ValueError(error_msg)
        else:
            error_msg = "Please set 'response_raw_output' property before trying to access its title value."
            self.logger.error(error_msg)
            raise ValueError(error_msg)

    @property
    def response_raw_output(self) -> str:
        return self._response_raw_output

    @response_raw_output.setter
    def response_raw_output(self, value):
        self._response_raw_output = ""
        try:
            if value:
                # Can occur under Linux when a curl response is received from a Windows server.
                if "\r" in value:
                    self.response_raw_output = value.replace("\r\n", "\n")  # Windows server responses
                    self.response_raw_output = self.response_raw_output.replace("\r", "\n")  # Mac OS <= 9, useless ?
                else:
                    self._response_raw_output = value
        except Exception as e:
            error_msg = f"Custom curl raw output parsing error. {e}"
            self.logger.error(error_msg)
            self.logger.error(repr(value))
            raise ValueError(error_msg)

    # *** Class identity, comparison and hashing functions *** #

    @classmethod
    def get_classname(cls) -> str:
        """Returns the fully-qualified class name string of this object."""
        return cls.__name__

    def use_classname(self) -> str:
        return self.get_classname()

    def __attrs(self):
        return (self.target_url, self.target_ip, self.bypass_mode, self.curl_base_options,
                self.request_curl_cmd, self.response_raw_output)

    def __eq__(self, other: Any) -> bool:
        """Equality test between objects, returns True if both objects have the same type and same attributes.

        :param CurlItem other: Other object to compare with this
        :return: True is same object, else False
        """
        return other.__class__ == CurlItem and self.__dict__ == other.__dict__

    def __ne__(self, other) -> bool:
        """Define a non-equality test.

        :param CurlItem other: Other object to compare with this
        :return: False is same object, else True
        """
        return not self.__eq__(other)

    def __hash__(self) -> int:
        """Used to get and store a unique object reference in immutable collections.

        :return: Object hash
        """
        return hash(str(self.__dict__))

    def __str__(self) -> str:
        out = ""
        out += f"Target url: {self.target_url.geturl()}\n"
        out += f"Target ip: {self.target_ip}\n"
        out += f"Bypass mode: {self.bypass_mode}\n"
        out += f"Curl command: {self.request_curl_cmd}\n"
        out += f"Curl base: {self.curl_base_options}\n"
        out += f"Curl payload: {self.request_curl_payload}\n"
        if self.response_raw_output:
            out += f"Formatted response: " \
                   f"{self.get_formatted_response(with_content_length=True, trunk_redirect_url=False)}\n"
        return out


class Tools:
    """Utility class for this project."""

    # Useful cross-platform properties
    IS_WINDOWS = platform.system() == "Windows"
    IS_LINUX = platform.system() == "Linux"
    SEPARATOR = "\\" if IS_WINDOWS else "/"

    @staticmethod
    def replacenth(string, sub, wanted, n) -> str:
        """Based on https://stackoverflow.com/a/35091558"""
        where = [m.start() for m in re.finditer(sub, string)][n - 1]
        return string[:where] + string[where:].replace(sub, wanted, 1)

    @staticmethod
    def get_new_logger(logger_name: str, with_colors=False, debug_level=False) -> logging.Logger:
        """Create and return new logger.

        :param str logger_name: Logger name
        :param with_colors: If True apply logger coloration with coloredlogs lib
        :param debug_level: If True, set level=logging.DEBUG else logging.INFO
        :return: Created logger object
        """
        new_logger = logging.getLogger(logger_name)
        if with_colors:
            coloredlogs.install(logger=new_logger, level=logging.DEBUG if debug_level else logging.INFO)
        else:
            new_logger.setLevel(logging.DEBUG if debug_level else logging.INFO)
            formatter = logging.Formatter("%(name)s %(levelname)s > %(message)s")
            handler = logging.StreamHandler()
            handler.setFormatter(formatter)
            new_logger.addHandler(handler)

        return new_logger

    @staticmethod
    def get_list_from_generic_arg(argument, arg_name="generic", enc_format=None, stdin_support=True,
                                  comma_string_support=True, ext_logger=None, debug=False) -> list[str]:
        """ Return list from generic argument.

        Useful when the argument could be a string, a list or a filename

        :param any argument: Argument value. Could be str / str list separated by comma / list / filename
        :param str arg_name: Argument name just for debug message (Optional)
        :param str enc_format: Encoding format to read file if arg is filename. Default (FR) = ISO-8859-1 (Optional)
        :param bool stdin_support: Make this function compatible with stdin '-' standard (default: True) (Optional)
        :param bool comma_string_support: Comma string "val1, val2,.." support in args (default: True) (Optional)
        :param logger ext_logger: Specify your own logger (default: None) (Optional)
        :param bool debug: Show method debug information (default:False, need ext_logger) (Optional)
        :return: List from generic argument
        """
        # STDIN manage (optional)
        if stdin_support and argument == "-":
            if ext_logger and debug:
                ext_logger.debug(f"Read '{arg_name}' argument as a list from stdin")
            return sys.stdin.read().strip().splitlines()
        # Arg value is already a list, useful for library mode. Just return
        if isinstance(argument, list):
            if ext_logger and debug:
                ext_logger.debug(f"The '{arg_name}' argument is already a list")
            return argument
        # Arg value is a filename (/path/file)
        elif os.path.isfile(argument):
            encoding = enc_format if enc_format else locale.getpreferredencoding(False)
            if ext_logger and debug:
                ext_logger.debug(f"The '{arg_name}' argument is a file")
            return [line.rstrip() for line in Tools.load_file_into_memory_list(
                argument, enc_format=encoding, clean_filename=False, external_file=True, return_str=False,
                ext_logger=ext_logger, debug=debug)]
        # Arg value is a string
        elif isinstance(argument, str):
            # Arg value is a string with multiple items separated by a comma ("value1, value2, ...")
            if comma_string_support and "," in argument:
                if ext_logger and debug:
                    ext_logger.debug(f"The '{arg_name}' argument is a string list separated by comma")
                return [arg.replace(" ", "") for arg in argument.split(",")]
            # Arg value is a simple string
            else:
                if ext_logger and debug:
                    ext_logger.debug(f"The '{arg_name}' argument is a string")
                return [argument]
        else:
            error_msg = f"The '{arg_name}' argument type is not supported"
            if ext_logger:
                ext_logger.error(error_msg)
            raise ValueError(error_msg)

    @overload
    @staticmethod
    def load_file_into_memory_list(filename: str, enc_format: str | None = None, clean_filename=False,
                                   external_file=False, return_str=False, debug=False, ext_logger=None) -> list[str]:
        ...

    @overload
    @staticmethod
    def load_file_into_memory_list(filename: str, enc_format: str | None = None, clean_filename=False,
                                   external_file=False, return_str=True, debug=False, ext_logger=None) -> str:
        ...

    @staticmethod
    def load_file_into_memory_list(filename: str, enc_format: str | None = None, clean_filename=False,
                                   external_file=False, return_str=False, debug=False,
                                   ext_logger=None) -> list[str] | str:
        """Load whole file into a list (or a string) in memory.

        Note: Fast for small files, do not use with huge file...

        :param str filename: Fast filename from args example: .\test.txt or "../../test.txt"
        :param str enc_format: Encoding format to read file. Default (locale.getpreferredencoding) (Optional)
        :param bool clean_filename: Is filename previously cleaned (transform_relative_filename) ? Default (False)
        :param bool external_file: If not clean_filename, don't use project root dir for absolute file name resolution
        :param bool return_str: Is true, this method return a string instead of a list
        :param bool debug: Show debug info from "transform_relative_filename" (Optional)
        :param logger ext_logger: Specify your own logger for debug (default: None) (Optional)
        :return: List or String (if return_str) containing the whole file
        """
        file = None
        encoding = enc_format if enc_format else locale.getpreferredencoding(False)

        # Transform relative path to absolute
        if not clean_filename:
            if external_file:
                # Absolute filename resolution based on the external file directory
                absolute_filename = \
                    os.path.join(os.path.realpath(os.path.dirname(filename)), os.path.basename(filename))
            else:
                # Absolute filename resolution of files located in project dir. Allow to call the tool from anywhere,
                # even with a symbolic link
                from importlib import resources
                try:
                    with resources.path(__name__, filename) as p:
                        absolute_filename = p
                except Exception:
                    absolute_filename = os.path.join(os.path.dirname(os.path.realpath(__file__)), filename)
            if ext_logger and debug and absolute_filename != filename:
                ext_logger.debug(f"Filename {filename} modified to {absolute_filename}")
        else:
            absolute_filename = filename
        # Not using with statement, prefer keep control of debug_class
        try:
            file = open(absolute_filename, encoding=encoding)
            if return_str:
                tmp = file.read()
            else:
                tmp = file.read().strip().splitlines()
        except OSError as error:
            if ext_logger:
                ext_logger.error(f"Unable to open {absolute_filename} file: \n{error}")
            raise
        except LookupError as error:
            error_msg = f"Unknown file encoding format '{encoding}' to read '{absolute_filename}' file: \n{error}"
            if ext_logger:
                ext_logger.error(error_msg)
            if file:
                file.close()
            raise
        else:
            file.close()
            return tmp

    @staticmethod
    def is_exist_directory(directory, force_create=False, create_mode=0o760) -> bool:
        res = os.path.isdir(directory)
        if not res and force_create:
            path = Path(directory)
            path.mkdir(mode=create_mode, parents=True, exist_ok=True)
            return True
        return res

    @staticmethod
    def get_path_with_url(base_path, url_obj) -> str:
        """ Generate a path combining the base path and URL components.

        :param str base_path: The base path to modify
        :param ParseResult url_obj: The parsed URL components
        :return: The combined path
        """
        # Format: output_dir / https - www.tld.com[-port -] - path - endpoint
        subdir_name = f"{url_obj.scheme}-{url_obj.netloc.replace(':', '-')}{url_obj.path.replace('/', '-')}"
        return f"{base_path}{Tools.SEPARATOR}{subdir_name.rstrip('-')}"


def main():
    # Show all options by Default
    if len(sys.argv) == 1:
        sys.argv.append("-h")
    arguments = docopt(__doc__, version=f"bypass_url_parser {__version__}")

    # Log level
    coloredlogs.install(logger=logger, level=logging.DEBUG if arguments["--debug"] else logging.INFO)

    # Debug args and config
    if arguments.get("--debug") == 2:
        logger.debug("=== Command line arguments ===")
        for key, val in arguments.items():
            logger.debug(f"{key}: {val}")

    # Init Bypasser object and run curl commands
    exporter = Bypasser(arguments, ext_logger=logger)
    exporter.run()


if __name__ == "__main__":
    main()
